# pip install requests 在项目的虚拟环境中添加依赖包
# requests 官方文档 https://requests.readthedocs.io/en/latest/
import requests
# pip install beautifulsoup4
# 官方文档 https://beautifulsoup.readthedocs.io/zh_CN/v4.4.0/
from bs4 import BeautifulSoup
# pip install lxml
from lxml import etree

url_bd = "http://www.baidu.com"
url_csdn = "https://bbs.csdn.net/"
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.80 Safari/537.36'}

if __name__ == '__main__':
    # request = requests.get(url=url_bd, headers=headers)
    # # print(request.status_code)
    # selector = etree.HTML(request.text)
    # house_list = selector.xpath("//*[@id='s-top-left']/a")
    # for house in house_list:
    #     print("title = " + house.xpath("text()")[0])
    #     print("url = " + house.xpath("@href")[0])
    #     print("\n")
    request = requests.get(url=url_csdn, headers=headers)
    selector = etree.HTML(request.text)
    # list = selector.xpath(
    #     "//div[@class='custom-tree-node']")
    # el-tree-node is-expanded is-focusable
    # menu_list = selector.xpath(
    #     '//div[contains(@class,"el-tree-node") and contains(@class,"is-expanded") and contains(@class,"is-focusable")]')
    menu_list = selector.xpath(
        '//div[contains(@class,"custom-tree-node")]')
    # contains(@class,"a") and contains(@class,"b")
    for item in menu_list:
        node_list = item.xpath("div[1]")
        child_list = item.xpath("div[2]")
        for node_item in node_list:
            print(node_item.xpath("div[1]/span[1]/text()")[0])
